#!/bin/sh
# PCP QA Test No. 1661
# Test pmproxy libpcp_web discovery file descriptor leaks
#
# Copyright (c) 2019 Red Hat.  All Rights Reserved.
#

seq=`basename $0`
echo "QA output created by $seq"

# get standard environment, filters and checks
. ./common.product
. ./common.filter
. ./common.check

[ $PCP_PLATFORM = linux ] || _notrun "Test only runs on Linux"
which pmproxy >/dev/null 2>&1 || _notrun "No pmproxy binary installed"
_check_series

_cleanup()
{
    echo;echo === cleaning up
    $sudo rm -rf $dir

    echo;echo === see $seq.full for logs 
    echo "+++ pmproxy.log +++" >>$seq.full
    cat $PCP_LOG_DIR/pmproxy/pmproxy.log >>$seq.full
    echo "+++ pmproxy.log.prev +++" >>$seq.full
    cat $PCP_LOG_DIR/pmproxy/pmproxy.log.prev >>$seq.full
    echo "+++ pmlogger.log +++" >>$seq.full
    cat $tmp.pmlogger.log >>$seq.full

    _restore_config $PCP_ETC_DIR/pcp/pmlogger
    _service pcp restart 2>&1 | _filter_pcp_stop | _filter_pcp_start
    _wait_for_pmcd
    _wait_for_pmlogger
    if $pmproxy_was_running
    then
	echo "Restart pmproxy ..." >>$here/$seq.full
	_service pmproxy restart >>$here/$seq.full 2>&1
	_wait_for_pmproxy
    else
	echo "Stopping pmproxy ..." >>$here/$seq.full
	_service pmproxy stop >>$here/$seq.full 2>&1
    fi
    $sudo rm -rf $tmp.*
}

_filter_pmproxy()
{
    sed \
	-e '/pmproxy: disabled time series, requires libuv support (missing)/d' \
    # end
}

_full_log()
{
    echo; echo $@ | tee -a $here/$seq.full
}

# get current log volume of primary pmlogger, according to pmproxy
#
_log_volume()
{
    path=`pminfo -f pmcd.pmlogger.archive | sed -n -e '/"primary"]/{
s/"$//
s/.*"//
p
}'`
    echo "primary pmlogger archive path: $path" >>$seq.full
    echo "contents of /proc/$pmproxy_pid/fd/ ..." >>$seq.full
    $sudo ls -l /proc/$pmproxy_pid/fd/ | \
        tee -a $seq.full | \
	grep "$path" | \
	$PCP_AWK_PROG -F. '/pmlogger.'$LOCALHOST'.*.[0-9]$/ {print $NF; exit}'
}

status=1	# failure is the default!
$sudo rm -rf $tmp $tmp.*
trap "_cleanup; exit \$status" 0 1 2 3 15

pmproxy_was_running=false
[ -f $PCP_RUN_DIR/pmproxy.pid ] && pmproxy_was_running=true
echo "pmproxy_was_running=$pmproxy_was_running" >>$here/$seq.full

# real QA test starts here
LOCALHOST=`hostname`

# only want the primary logger running
_save_config $PCP_ETC_DIR/pcp/pmlogger
_restore_pmlogger_control
_service pmlogger stop 2>&1 | _filter_pcp_stop

_full_log === restarting pmproxy service to ensure sane starting condition 
_service pmproxy restart 2>&1 | _filter_pcp_stop | _filter_pcp_start | _filter_pmproxy
_wait_for_pmproxy

pmproxy_pid=`_get_pids_by_name -a pmproxy`
[ -z "$pmproxy_pid" ] && echo === pmproxy not running && exit
date >>$seq.full
$PCP_PS_PROG $PCP_PS_ALL_FLAGS | egrep '[P]ID|[p]mproxy' >>$seq.full
echo "pmproxy_pid=$pmproxy_pid" >>$seq.full

_full_log === creating archive in new archive directory 
dir=$PCP_ARCHIVE_DIR/qa1661
$sudo sh -c ". $PCP_SHARE_DIR/lib/rc-proc.sh; mkdir_and_chown $dir 775 $PCP_USER:$PCP_GROUP"
# make sure no archive files left here from a previous run of this test
$sudo rm -f $dir/testarchive.*
$sudo sh -c "pmlogger -s 20 -t 1 -c config.default -l$tmp.pmlogger.log $dir/testarchive"

_full_log === checking pmproxy file descriptors for deleted files, should be none 
$sudo ls -l /proc/$pmproxy_pid/fd | grep 'testarchive.*(deleted)'

_full_log === compressing the new archives, which also deletes the originals 
for suff in 0 meta
do
    if [ -f $dir/testarchive.$suff ]
    then
	$sudo xz $dir/testarchive.$suff
    else
	echo "Error: $dir/testarchive.$suff missing!"
	ls -l $dir
    fi
    $sudo chown -R $PCP_USER $dir
done

_full_log === restarting pmlogger # primary only
_service pmlogger restart 2>&1 | _filter_pcp_start
_wait_for_pmlogger

_full_log === wait for pmproxy to process filesystem events
pmsleep 5

_full_log === checking pmproxy file descriptors for deleted files, should be none 
$sudo ls -l /proc/$pmproxy_pid/fd | grep 'testarchive.*(deleted)'

_full_log === checking pmproxy correctly switches logvol when pmlogger bumps to a new logvol
curlogvol=`_log_volume`
echo "curlogvol=$curlogvol" >>$seq.full
if [ -z "$curlogvol" ]
then
    echo "Arrgh ... curlogvol not found from ..."
    $sudo ls -l /proc/$pmproxy_pid/fd/
    echo "pmproxy_pid=$pmproxy_pid"
    $PCP_PS_PROG $PCP_PS_ALL_FLAGS | egrep '[P]ID|[p]mproxy'
    exit
fi
echo 'new volume' | $sudo pmlc -P >> $here/$seq.full 2>&1
# make sure pmproxy has noticed ...
sleep 1
newlogvol=`_log_volume`
echo "newlogvol=$newlogvol" >>$seq.full
if [ -z "$newlogvol" ]
then
    echo "Arrgh ... newlogvol not found from ..."
    $sudo ls -l /proc/$pmproxy_pid/fd/
    echo "pmproxy_pid=$pmproxy_pid"
    $PCP_PS_PROG $PCP_PS_ALL_FLAGS | egrep '[P]ID|[p]mproxy'
    exit
fi
if [ `expr $curlogvol + 1` -ne $newlogvol ]
then
    echo FAILED curlogvol=$curlogvol newlogvol=$newlogvol
else
    echo passed
fi

_full_log === checking pmproxy survives pmlogger restart
_service pmlogger restart 2>&1 | _filter_pcp_start
_wait_for_pmlogger
$sudo [ ! -d /proc/$pmproxy_pid/fd/ ] && echo FAIL && exit
echo passed

# success, all done
status=0
exit
